package utils;

import java.io.*;
import java.text.Collator;
import java.util.*;

public class StopwordsCombine {

    public static void main(String[] args) {
        BufferedReader bswReader;
        BufferedReader hswReader;
        List<String> stopWords = new ArrayList<String>();
        try {
             bswReader = new BufferedReader(new InputStreamReader(new FileInputStream("D:/bsw.txt"), "UTF-8"));
             hswReader = new BufferedReader(new InputStreamReader(new FileInputStream("D:/hsw.txt"), "UTF-8"));
             String word;
             while ((word = bswReader.readLine()) != null) {
                 stopWords.add(word);
             }
             while ((word = hswReader.readLine()) != null) {
                 stopWords.add(word);
             }
        } catch (IOException e) {
            e.printStackTrace();
            return;
        }
        for(int i = 65; i < 91; i++) {
            char ch = (char)i;
            String str = String.valueOf(ch);
            stopWords.add(str);
        }
        for(int i = 97; i < 123; i++) {
            char ch = (char)i;
            String str = String.valueOf(ch);
            stopWords.add(str);
        }
        stopWords = new ArrayList<String>(new HashSet<String>(stopWords));
        Collections.sort(stopWords ,new Comparator<String>(){
            Collator clt=Collator.getInstance();

            public int compare(String o1, String o2) {
                return clt.compare(o1, o2);
            }
        });
        BufferedWriter writer = null;
        try {
             writer = new BufferedWriter(new FileWriter("D:/stopwords.txt"));
            for (int i = 0; i < stopWords.size(); i++) {
                writer.write(stopWords.get(i));
                writer.newLine();
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                writer.flush();
                writer.close();
            } catch (IOException e) {
                e.printStackTrace();
            }

        }


    }
}
